import requests
from lxml import html
from pymongo import MongoClient

# 连接到 MongoDB
client = MongoClient('mongodb://localhost:27017/')
db = client['mydb1']
collection = db['quotes']

session = requests.Session()
data = {
    "csrf_token": "JuUZjLcpehvHAlFEIzXgPKnOxCsQWfytMDkYiRSNGrVBomTadbqw",
    "username": "123",
    "password": "123"
}

login_url = "https://quotes.toscrape.com/login"

res = session.post(login_url, data=data)
print(res.cookies)

all_data = []

url = "https://quotes.toscrape.com/"
while True:
    response = session.get(url)
    tree = html.fromstring(response.text)
    # 提取名言、作者和标签信息
    quotes = tree.xpath('//div[@class="quote"]')

    for quote in quotes:
        text = quote.xpath('.//span[@class="text"]/text()')[0]
        author = quote.xpath('.//small[@class="author"]/text()')[0]
        tags = quote.xpath('.//a[@class="tag"]/text()')
        quote_data = {
            'text': text,
            'author': author,
            'tags': tags
        }
        all_data.append(quote_data)
        try:
            # 将数据插入到 MongoDB 中
            collection.insert_one(quote_data)
        except Exception as e:
            print(f"数据插入出错: {e}")

    # 查找下一页的链接
    next_page = tree.xpath('//li[@class="next"]/a/@href')
    if next_page:
        url = "https://quotes.toscrape.com/" + next_page[0]
    else:
        break

for data in all_data:
    print(data)

print("数据已保存到 MongoDB 中")

# 关闭 MongoDB 连接
client.close()
